'''
Created on Sep 8, 2015

@author: eyaomai
'''
import sys
sys.path.append(sys.argv[0][:sys.argv[0].rfind('com/stocklens')])

from com.stocklens.stock.common.utils import Logging,  PIDUtils, CTPUtils
from com.stocklens.stock.data.crawlercommon import CrawlerConstants, CrawlerManager, CrawlerBase
from com.stocklens.stock.data.sina.sinacommon import SinaConstants
from bs4 import BeautifulSoup
import traceback
import datetime

class SinaMarginTradeMgr(CrawlerManager):
    LOGGER_NAME_CRAWL_SMT = 'smt'
    FILE_CONFIG_DATERANGE = 'dateRange'
    FILE_CONFIG_STARTDATE = 'startDate'
    FILE_CONFIG_ENDDATE = 'endDate'
    def __init__(self, json_config_file):
        '''
        Constructor
        '''
        super(SinaMarginTradeMgr, self).__init__(json_config_file, 0.1, None)
        self.logger = Logging.getLogger(SinaMarginTradeMgr.LOGGER_NAME_CRAWL_SMT)

    def start(self):
        super(SinaMarginTradeMgr, self).start()

    def _generateTask(self, task, checkTaskList=True):
        if super(SinaMarginTradeMgr, self)._generateTask(task, checkTaskList) is False:
            return False
         
        task[CrawlerConstants.PARA_CLASS] = SinaMarginTradeCrawler
        task[SinaMarginTradeCrawler.PARA_TRADE_DATE] = self._taskList.pop(0)
        return True

    def _initTask(self):
        now = datetime.date.today()
        beginDate = datetime.date(now.year, 1,1)
        sql = 'SELECT max(date) from sina_margin_trading_detail'
        if self.dbProxy.execute(sql)>0:
            result = self.dbProxy.cur.fetchone()
            if result[0] is not None:
                beginDate = result[0]+datetime.timedelta(days=1)
        while beginDate<=now:
            if not CTPUtils.Singleton.isHoliday(beginDate):
                self._taskList.append(beginDate.strftime("%Y-%m-%d"))
            beginDate += datetime.timedelta(days=1)   
        
        self.logger.debug('Task list:%s', self._taskList)

class SinaMarginTradeCrawler(CrawlerBase):
    PARA_TRADE_DATE = 'tradedate'
    URL = 'http://vip.stock.finance.sina.com.cn/q/go.php/vInvestConsult/kind/rzrq/index.phtml?tradedate=%s'
    def __init__(self, controller, dbProxy, request):
        super(SinaMarginTradeCrawler, self).__init__(controller, dbProxy, request)
        self.__url = None
        self.__tradedate = None
        self.__parsePara(request)
    
    def __parsePara(self, request):
        self.__tradedate = request[SinaMarginTradeCrawler.PARA_TRADE_DATE]
        self.__url = SinaMarginTradeCrawler.URL % self.__tradedate
    
    def run(self):
        super(SinaMarginTradeCrawler, self).run()
        status = self.__crawlData()
        self._reportDone(status)
    
    def __crawlData(self):
        self.logger.info('Begin to Crawl %s', self.__url)
        status = CrawlerConstants.VAL_STATUS_FINISH
        try:            
            content = self._fetchContent(self.__url)
            if content is None:
                status = CrawlerConstants.VAL_STATUS_FAILURE
            else:
                marginTradeDataList = self.__parseMarginTradeData(content)
                if marginTradeDataList is not None and len(marginTradeDataList)>0:
                    self.__writeDb(marginTradeDataList)
        except Exception:
            traceInfo = traceback.format_exc()
            self.logger.warn('Fail to Crawl %s:%s', self.__url, traceInfo)
            status = CrawlerConstants.VAL_STATUS_FAILURE
        self.logger.info('Finish to Crawl %s', self.__url)
        return status      

    def __parseMarginTradeData(self, content):
        lindex = content.rfind('<table class="list_table" id="dataTable">')
        if lindex<0:
            self.logger.warn('No dataTable for %s', self.__url)
            return None
        soup = BeautifulSoup(content[lindex:])
        tables = soup.findAll('table',{'id':'dataTable'})
        if len(tables) == 0:
            Logging.LOGGER.error('No dataTable found for url:%s', self.__url)
            return None
        trs = tables[0].findAll('tr')
        if len(trs)<3:
            return None
        marginTradeDataList = list()
        for tr in trs[3:]:
            tds = tr.findAll('td')
            marginTradeDataList.append(MarginTradeData(
                                       tds[1].text.strip(),#stockid
                                       self.__convertNumericData(tds[3], float),
                                       self.__convertNumericData(tds[4], float),
                                       self.__convertNumericData(tds[6], float),
                                       self.__convertNumericData(tds[7], int),
                                       self.__convertNumericData(tds[8], int),
                                       ))
        
        return marginTradeDataList
    
    def __convertNumericData(self, td, ctype):
        text = td.text.strip()
        if text == '--':
            return -1
        if ctype == int:
            index = text.find('.')
            if index>=0:
                text = text[:index]
            text = text.replace(',','')
        elif ctype == float:
            text = text.replace(',','')
        return ctype(text)
    def __writeDb(self, marginTradeDataList):
        sql = 'INSERT INTO sina_margin_trading_detail (stockid, market, date, financing_remaining_amount,financing_purchase_amount,loan_remaining_amount,loan_remaining_vol,loan_sell_vol) values '
        values = list()
        #self.logger.debug('start:%d, end:%d', start, end)
        #for hisData in hisDataList[start:end]:
        for marginTradeData in marginTradeDataList:
            valuestr = '("%s","%s", "%s", %f, %f, %f, %d, %d)' %\
                            (
                             marginTradeData.stockid,
                             marginTradeData.market,
                             self.__tradedate,
                             marginTradeData.financing_remaining_amount,
                             marginTradeData.financing_purchase_amount,
                             marginTradeData.loan_remaining_amount,
                             marginTradeData.loan_remaining_vol,
                             marginTradeData.loan_sell_vol
                             )
            values.append(valuestr)
        sql = sql + ','.join(values)
        if self.dbProxy.execute(sql)>0:
            self.totalNum+=len(values)
            self.dbProxy.commit()
        else:
            return False
        return True
    
class MarginTradeData(object):
    def __init__(self,
                 stockid, 
                 financing_remaining_amount,
                 financing_purchase_amount,
                 loan_remaining_amount,
                 loan_remaining_vol,
                 loan_sell_vol):
        self.stockid = stockid
        if stockid.startswith('6'):
            self.market = 'SZ'
        else:
            self.market = 'SH'
        self.financing_remaining_amount = financing_remaining_amount
        self.financing_purchase_amount = financing_purchase_amount
        self.loan_remaining_amount = loan_remaining_amount
        self.loan_remaining_vol = loan_remaining_vol
        self.loan_sell_vol = loan_sell_vol
if __name__ == '__main__':
    if PIDUtils.isPidFileExist('smt'):
        print 'Previous Sina Magin Trading History Crawler process is on-going, please stop it firstly'
        sys.exit(1)
    import os
    pid = os.getpid()
    PIDUtils.writePid('smt', pid)
    Logging.initLogger('conf/crawler/crawler.logging.cfg')
    smt = SinaMarginTradeMgr('conf/crawler/smt.cfg')
    smt.start()
    pidutils = PIDUtils('smt', smt.shutDown, 5, smt.logger)
    pidutils.start()
    sys.exit(0)
                        